package com.lizhiyu.flink.demo3_sink.kafka;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;

import java.util.Properties;

/***
 * 使用flink 官方插件,将kafka上的信息获得下来，然后再发送到另一个kafka队列上
 *
 * 使用前要创建zk 和 kafka
 * kafka 中创建两个topic,一个程序source接收其中信息 一个程序sink发送到其中
 */
public class FlinkKafkaConnector {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        Properties props = new Properties();
        //kafka地址
        props.setProperty("bootstrap.servers", "39.107.109.94:9092");
        //组名
        props.setProperty("group.id", "flink");
        //字符串序列化和反序列化规则
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        //offset重置规则
        props.setProperty("auto.offset.reset", "latest");
        //自动提交
        props.setProperty("enable.auto.commit", "true");
        props.setProperty("auto.commit.interval.ms", "2000");
        //有后台线程每隔10s检测一下Kafka的分区变化情况
        props.setProperty("flink.partition-discovery.interval-millis","10000");

        FlinkKafkaConsumer<String> consumer =new FlinkKafkaConsumer<>("flink-topic", new SimpleStringSchema(), props);

        //设置从记录的消费者组内的offset开始消费
        consumer.setStartFromGroupOffsets();

        DataStream<String> ds = env.addSource(consumer);
        ds.print();
        SingleOutputStreamOperator<String> mapDs = ds.map(new MapFunction<String, String>() {
            @Override
            public String map(String value) throws Exception {
                return "flink" + value;
            }
        });
        FlinkKafkaProducer<String> flinkKafkaProducer = new FlinkKafkaProducer<String>("flink2-topic", new SimpleStringSchema(), props);
        mapDs.addSink(flinkKafkaProducer);
        env.execute();
    }
}
